# -*- coding: utf-8 -*-
import scrapy
import employment.items as items
import re
import time
import urllib


class A51jobSpider(scrapy.Spider):
    name = '51job'
    allowed_domains = ['search.51job.com']
    #start_urls = [r'https://search.51job.com/list/000000,000000,0000,32,9,99,C%252B%252B,2,105.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=']
    #start_urls = [r'https://search.51job.com/list/000000,000000,0000,40,9,99,C%252B%252B,2,1.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=']
    start_urls = [r'https://search.51job.com/list/000000,000000,0000,01%252c37%252c38%252c31%252c32,9,99,C%252B%252B,2,2.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=']

    def parse(self, response):

        # 获取当前页的职位所有内容

        positionList = response.css(".j_joblist .e")
        for pL in positionList:
            item = items.EmploymentItem()
            # 职位名称
            item['positionName'] = pL.css("a.el .jname::text").extract_first()
            # 职位薪酬

            # item['positionAward'] = pL.css(
            #     "a.el .info .sal::text").extract_first()

            # positionAward = pL.css(
            #     "a.el .info .sal::text").extract_first()
            # matchObj = re.search(
            #     r'([0-9.]*?)-([0-9.]*?)万/月', str(positionAward))
            # if matchObj != None:
            #     item['positionAwardMin'] = matchObj.group(1)
            #     item['positionAwardMax'] = matchObj.group(2)
            # else:
            #     raise RuntimeError('error positionAward:' + positionAward)

            # item['positionAwardMax'] = pL.css("a.el .info .sal::text").extract_first()
            positionAward = '0-0千/月'
            if pL.css("a.el .info .sal::text").extract_first() != None:

                positionAward = pL.css("a.el .info .sal::text").extract_first()
            else:
                item['positionAwardMin'] = 0
                item['positionAwardMax'] = 0
            matchObj1 = re.search(
                r'([0-9.]*?)-([0-9.]*?)千/月', str(positionAward))
            matchObj2 = re.search(
                r'([0-9.]*?)-([0-9.]*?)万/月', str(positionAward))
            matchObj3 = re.search(
                r'([0-9.]*?)-([0-9.]*?)万/年', str(positionAward))
            matchObj4 = re.search(r'([0-9.]*?)元/天', str(positionAward))
            if matchObj1 != None:
                item['positionAwardMin'] = matchObj1.group(1)
                item['positionAwardMax'] = matchObj1.group(2)
            elif matchObj2 != None:
                item['positionAwardMin'] = float(matchObj2.group(1))*10
                item['positionAwardMax'] = float(matchObj2.group(2))*10
            elif matchObj3 != None:
                item['positionAwardMin'] = float(matchObj3.group(1))*10/12
                item['positionAwardMax'] = float(matchObj3.group(2))*10/12
            elif matchObj4 != None:
                item['positionAwardMin'] = float(matchObj4.group(1))*27/1000
                item['positionAwardMax'] = float(matchObj4.group(1))*31/1000
            else:
                raise RuntimeError('type of positionAward:' +
                                   type(positionAward)+positionAward)
            # 发布时间
            positionTime = pL.css("a.el .time::text").extract_first()
            matchTime = re.search(
                r'([0-9.]*?)-([0-9.]*?)发布', str(positionTime))
            new_date = '2020-'+matchTime.group(1) + '-' + matchTime.group(2)
            st = time.strptime(new_date, '%Y-%m-%d')
            update_date = time.strftime('%Y-%m-%d', st)
            item['positionTime'] = update_date
            str1 = pL.css("a.el .info .d::text").extract_first()
            # 工作经验要求

            # experienceTime = str1.split('|')[1]
            # if experienceTime.strip() == '经验不限':
            #     # 若min与max都为-1，则代表经验不限
            #     item['experienceTimeMin'] = -1
            #     item['experienceTimeMax'] = -1
            # elif experienceTime.strip() == '在校生/应届生':
            #     # 若min与max都为0，则代表为应届生
            #     item['experienceTimeMin'] = 0
            #     item['experienceTimeMax'] = 0
            # else:
            #     matchObj = re.search(
            #         r'([0-9.]*?)-([0-9.]*?)年经验', str(experienceTime))
            #     if (matchObj) != None:
            #         # 例： '3-5年经验'
            #         item['experienceTimeMin'] = int(matchObj.group(1))
            #         item['experienceTimeMax'] = int(matchObj.group(2))
            #     else:
            #         matchObj = re.search(
            #             r'经验([0-9.]*?)年以上', str(experienceTime))
            #         if (matchObj) != None:
            #             # 若max为-1，代表不限
            #             # 例： '经验10年以上'
            #             item['experienceTimeMin'] = matchObj.group(1)
            #             item['experienceTimeMax'] = -1
            #         else:
            #             matchObj = re.search(
            #                 r'经验([0-9.]*?)年以下', str(experienceTime))
            #             if (matchObj) != None:
            #                 # 还真有这种奇葩岗位要求经验n年以下的
            #                 # 若min为-1，代表不限
            #                 item['experienceTimeMin'] = -1
            #                 item['experienceTimeMax'] = matchObj.group(1)
            #             else:
            #                 raise RuntimeError('新的岗位经验要求:'+experienceTime)

           # item['experienceTimeMin'] = str1.split('|')[1]
            exp = '不限'
            if str1.split('|')[1] != None:
                exp = str1.split('|')[1]
            else:
                item['experienceTimeMin'] = -1
                item['experienceTimeMax'] = -1
            print(exp)
            if exp == '不限':
                # 若min与max都为-1，则代表经验不限
                item['experienceTimeMin'] = -1
                item['experienceTimeMax'] = -1
            elif exp == '经验在校/应届':
                # 若min与max都为0，则代表为应届生
                item['experienceTimeMin'] = 0
                item['experienceTimeMax'] = 0
            elif (matchObj: = re.search(r'([0-9.]*?)-([0-9.]*?)年经验', str(exp))) != None:
                item['experienceTimeMin'] = matchObj.group(1)
                item['experienceTimeMax'] = matchObj.group(2)
            elif (matchObj: = re.search(r'([0-9.]*?)年以上经验', str(exp))) != None:
                # 若max为-1，代表不限
                # 例： '经验10年以上'
                item['experienceTimeMin'] = matchObj.group(1)
                item['experienceTimeMax'] = -1
            elif (matchObj: = re.search(r'([0-9.]*?)年经验', str(exp))) != None:
                # 若max为-1，代表不限
                # 例： '经验10年以上'
                item['experienceTimeMin'] = matchObj.group(1)
                item['experienceTimeMax'] = matchObj.group(1)

            else:
                item['experienceTimeMin'] = -1
                item['experienceTimeMax'] = -1
            # 公司所在城市
            str2 = str1.split('|')[0]
            if (matchCity: = re.match(r'(.*)-(.*?)', str(str2))) != None:
                item['ltdCity'] = str2.split('-')[0]+'.'+str2.split('-')[1]
            else:
                item['ltdCity'] = str2 = str1.split('|')[0]
            # 学历要求
            edu_map = {
                '不限': 0,
                '大专': 2,
                '本科': 3,
                '硕士': 4,
                '博士': 5,
                '中技': 6
            }
            edu = '不限'
            length = len(set(str1.split('|')))
            if length > 3:
                edu = str1.split('|')[2]
                item['education'] = edu_map.get(edu.strip(), -1)
            else:
                item['education'] = 0

            #item['education'] = str1.split('|')[2]
            # 福利待遇
            item['walfare'] = pL.css("a.el .tags::attr(title)").extract_first()
            # 公司名称
            item['ltdName'] = pL.css(".er .cname::attr(title)").extract_first()
            str1 = pL.css(".er .dc::text").extract_first()
            # 公司规模
            leng = len(set(str1.split('|')))
            if leng > 1:
                item['ltdScale'] = str1.split('|')[1]
            else:
                item['ltdScale'] = '50-100人'
            # 职位详情链接
            item['positionLink'] = pL.css(
                ".er .cname::attr(href)").extract_first()
            url = pL.css("a.el::attr(href)").extract_first()
            # 公司链接
            item['ltdLink'] = urllib.parse.urljoin(response.url, url)  # 拼接url
            yield item
         # 翻页
        # 获取总页数

        curNum = response.css("li.on div::text").extract_first()

       # if curNum==None or totalNum==None:
        #   raise RuntimeError('无法获取页数')

        curNum = int(curNum)
        totalNum = 233

        if curNum+1 <= totalNum:
            print('==================================================测试=============================================')
            print(curNum)
            print(totalNum)
            print('==================================================测试=============================================')
            next_page_css = "li.next a"

            yield scrapy.Request(response.url, callback=self.parse, meta={
                'next_page_css': next_page_css,
                # 'next_page_num':curNum+1,# 解决另一个问题：总是回到第一页，然后翻页到第二页就停下来了
            }, dont_filter=True)  # 注意防止过滤重复请求

    def nextPage(self, url, pageno):
        '''
        翻页，返回下一页的Request
        @param url: 本页的url
        @param pageno: 页码
        '''
        return scrapy.http.FormRequest(url, formdata={
            'pn': str(pageno),
            'kd': '',
            'first': 'true',
        }, callback=self.parse)

        # 获取下一页的链接
    #     next_page_url = self.getNextPageUrl(response)

    #     if next_page_url != None:
    #         # 如果还有下一页，则继续爬取下一页
    #         yield scrapy.Request(next_page_url, callback=self.parse)

    # def getNextPageUrl(self, response, next_page_num=None):
    #     '''
    #     返回下一页的url
    #     @param next_page_num: 指定下一页的页码，默认为当前页的下一页
    #     @return 下一页的url，如果下一页的页码超出范围，则返回None
    #     '''

    #     # 获取当前板块有多少页
    #     j_page = response.css(".j_page")[0]
    #     #page_sum_str = j_page.css("span.td::text").extract_first()
    #     page_sum = 154
    #     # 获取当前页数
    #     current_page_num = int(j_page.css("li.on div::text")[0].extract())
    #     print("-----------------------当前页数----------------------")
    #     print(current_page_num)
    #     # 修改当前url为下一页的url
    #     if next_page_num == None:
    #         next_page_num = current_page_num + 1
    #     if next_page_num > page_sum or next_page_num <= 0:
    #         return None
    #     current_page_url = response.url
    #     next_page_url = re.sub(",\d.html", ",{}.html".format(
    #         next_page_num), current_page_url)
    #     return next_page_url

# 教学
